package ca.uwindsor.cs.deepweb.utility.chinesenovel;

import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.util.TreeSet;

import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.index.CorruptIndexException;
import org.apache.lucene.store.LockObtainFailedException;

import ca.uwindsor.cs.deepweb.estimation.DictionariesFilesContentIndexer;
import ca.uwindsor.cs.deepweb.estimation.FilesContentIndexer;
import ca.uwindsor.cs.deepweb.utility.NekoHTMLHandler;
import ca.uwindsor.cs.deepweb.utility.TxtDocumentHandler;

public class MixedFilesContentIndexer extends DictionariesFilesContentIndexer {

	protected TreeSet<String> blacklist;

	public MixedFilesContentIndexer(String sourcepath, String indexdir,
			String encoding, boolean verbose, boolean create, String[] ext) {
		super(sourcepath, indexdir, encoding, verbose, create, ext);
		blacklist = new TreeSet<String>();
		blacklist.add("default.html");
		blacklist.add("default.htm");
		blacklist.add("index.html");
		blacklist.add("index.htm");
		blacklist.add("index.xml");
		blacklist.add("filelist.xml");
	}

	protected void generateIndex(File dir) {
		try {
			File[] dataFiles = dir.listFiles();
			File file;
			int size;
			// Add documents to the index
			for (int i = 0; i < dataFiles.length; i++) {
				file = dataFiles[i];
				if (file.isFile()) {
					if (isValidExtension(file.getName()) && !blacklist.contains(file.getName())) {
							String content,title;
							if (file.getName().endsWith(".htm")
									|| file.getName().endsWith(".html")
									|| file.getName().endsWith(".xml")) {
								NekoHTMLHandler handler = new NekoHTMLHandler(
										encoding);
								
								try {
									handler.processContent(new FileInputStream(
											dataFiles[i]));
								} catch (Exception e) {
									// e.printStackTrace();
									System.err.println("Error on parsing: "
											+ dataFiles[i].getCanonicalPath());
									continue;
								}
								content=handler.getContent().replaceAll(
										"\\s+", " ");
								title=handler.getTitle();
							} else{
								TxtDocumentHandler handler = new TxtDocumentHandler(encoding);
								try {
									handler.processContent(new FileInputStream(
											dataFiles[i]));
								} catch (Exception e) {
									// e.printStackTrace();
									System.err.println("Error on parsing: "
											+ dataFiles[i].getCanonicalPath());
									continue;
								}
								content=handler.getContent().replaceAll("+", "");
								title=handler.getTitle();
							}

							size = content.length();

							if (isVerbose) {
								System.out.println("Indexing file "
										+ file.getCanonicalPath());
							}

							Document document = new Document();
							// make the ID from the path shorter - get the file
							// name
							String word = new String("ChineseNovel"+globalID);

							// String word = String.valueOf(globalID);
							// System.out.println(id);

							Field FieldPath = new Field(
									FilesContentIndexer.FIELD_ID, word,
									Field.Store.YES, Field.Index.UN_TOKENIZED);

							Field FieldTitle = new Field(
									FilesContentIndexer.FIELD_TITLE, title, Field.Store.YES,
									Field.Index.UN_TOKENIZED);

							Field FieldSize = new Field(
									FilesContentIndexer.FIELD_SIZE, Integer
											.toString(size), Field.Store.YES,
									Field.Index.UN_TOKENIZED);

							Field FieldBody = new Field(
									FilesContentIndexer.FIELD_CONTENT, content,
									Field.Store.NO, Field.Index.TOKENIZED,
									Field.TermVector.WITH_POSITIONS_OFFSETS);

							document.add(FieldPath);
							document.add(FieldTitle);
							document.add(FieldSize);
							document.add(FieldBody);
							indexWriter.addDocument(document);
							// txtReader.close();
							globalID++;
					}
				} else if (file.isDirectory()) {
					generateIndex(dataFiles[i]);
				}
			}
		} catch (CorruptIndexException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
			System.exit(1);
		} catch (LockObtainFailedException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
			System.exit(1);
		} catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
			System.exit(1);
		}
	}

	public static void main(String[] args) {
		// String[] ext = { "html" };
		// HTMLFilesContentIndexer indexer = new HTMLFilesContentIndexer(
		// "G:/gov2-corpus-extracted", "F:/indexes/GOV2", "UTF-8", true,
		// true, ext);
		// indexer.generateIndex();

		// args = new String[7];
		// args[0] = "--SourceDir=H:\\gov2-corpus-extracted\\GX000\\00";
		// args[1] = "--IndexDir=F:\\indexes\\GOV2_lim2048b";
		// args[2] = "--encoding=latin1";
		// args[3] = "-verbose";
		// args[4]= "-create";
		// args[5]= "-resursive";
		// args[6]= "--extension=html";

		String SourceDir = new String();
		String IndexDir = new String();
		String encoding = new String("UTF-8");
		boolean isVerbose = false;
		boolean isCreate = true;
		boolean isRecursive = false;

		String ext[] = new String[] { "html" };

		if (args.length < 2) {
			System.out
					.println("usage: java -jar HTMLFilesContentIndexer --SourceDir=sourcefiledirectory --IndexDir=indexfiledirectory --extension=txt,html,... [--encoding=encoding](default=UTF-8) [-verbose] [-create] !");
			System.exit(0);
		} else {
			for (int i = 0; i < args.length; i++) {
				if (args[i].startsWith("--SourceDir=")) {
					SourceDir = args[i].substring(12);
				} else if (args[i].startsWith("--IndexDir=")) {
					IndexDir = args[i].substring(11);
				} else if (args[i].startsWith("--encoding=")) {
					encoding = args[i].substring(11);
				} else if (args[i].equals("-create")) {
					isCreate  = true;
				} else if (args[i].equals("-verbose")) {
					isVerbose= true;
				} else if (args[i].startsWith("--extension=")) {
					String tempexts = args[i].substring(12);
					ext = tempexts.split(",");
				} else if (args[i].endsWith("-resursive")) {
					isRecursive = true;
				}
			}
		}

		if (SourceDir.equals("") || IndexDir.equals("")) {
			System.out
					.println("usage: java -jar HTMLFilesContentIndexer SourceDir=sourcefiledirectory IndexDir=indexfiledirectory [encoding=encoding](default=UTF-8) [-verbose]!");
			System.exit(0);
		} else {
			MixedFilesContentIndexer indexer;
			// if (isRecursive) {
			// indexer = new HTMLFilesContentIndexer(SourceDir, IndexDir,
			// encoding,
			// isVerbose, isCreate, ext);
			// } else {
			// indexer = new HTMLFilesContentIndexer (SourceDir, IndexDir,
			// encoding,
			// isVerbose, isCreate, ext);
			// }
			indexer = new MixedFilesContentIndexer(SourceDir, IndexDir,
					encoding, isVerbose, isCreate, ext);
			indexer.generateIndex();
		}

	}
}
